"""重构后的知乎爬虫
使用新的架构实现
"""
import json
import logging
from typing import Dict, Any, List, Optional
from datetime import datetime
from bs4 import BeautifulSoup

from .enhanced_base_spider import EnhancedBaseSpider
from ..models.hot_item import HotItem, HotItemStatus, Platform, HotItemBatch
from ..utils.config_manager import config_manager

logger = logging.getLogger(__name__)


class RefactoredZhihuSpider(EnhancedBaseSpider):
    """重构后的知乎爬虫"""
    
    def __init__(self):
        """初始化知乎爬虫"""
        super().__init__(platform='zhihu', name='RefactoredZhihuSpider')
        
        # 添加知乎特定的中间件
        self.add_middleware(self._add_zhihu_headers)
    
    async def _add_zhihu_headers(self, url: str, kwargs: Dict[str, Any]) -> Dict[str, Any]:
        """添加知乎特定的请求头
        
        Args:
            url: 请求URL
            kwargs: 请求参数
            
        Returns:
            修改后的请求参数
        """
        if 'headers' not in kwargs:
            kwargs['headers'] = {}
        
        # 知乎API需要的特殊请求头
        kwargs['headers'].update({
            'X-Zse-93': '101_3_3.0',
            'X-Zse-96': '2.0',
            'X-App-Za': 'OS=Web'
        })
        
        return kwargs
    
    async def parse(self, html: str, url: str = None) -> List[Dict[str, Any]]:
        """解析知乎热榜数据
        
        Args:
            html: 响应内容
            url: 请求URL
            
        Returns:
            解析后的数据列表
        """
        items = []
        
        try:
            # 判断是API响应还是HTML页面
            if html.strip().startswith('{'):
                # JSON响应
                items = await self._parse_json_response(html, url)
            else:
                # HTML页面
                items = await self._parse_html_response(html, url)
        except Exception as e:
            logger.error(f"Parse error: {e}")
            
        return items
    
    async def _parse_json_response(self, json_str: str, url: str = None) -> List[Dict[str, Any]]:
        """解析JSON响应
        
        Args:
            json_str: JSON字符串
            url: 请求URL
            
        Returns:
            解析后的数据列表
        """
        try:
            data = json.loads(json_str)
        except json.JSONDecodeError:
            logger.error("Failed to parse JSON response")
            return []
        
        items = []
        
        # 检查响应格式
        if 'data' in data:
            hot_list = data.get('data', [])
        elif 'hot_list' in data:
            hot_list = data.get('hot_list', [])
        else:
            logger.warning("Unknown response format")
            return []
        
        for idx, item in enumerate(hot_list, 1):
            try:
                # 提取目标信息
                target = item.get('target', {})
                
                # 判断类型
                item_type = target.get('type', '')
                
                if item_type == 'question':
                    hot_item = self._parse_question_item(idx, item, target)
                elif item_type == 'answer':
                    hot_item = self._parse_answer_item(idx, item, target)
                elif item_type == 'article':
                    hot_item = self._parse_article_item(idx, item, target)
                else:
                    hot_item = self._parse_generic_item(idx, item, target)
                
                if hot_item:
                    items.append(hot_item)
                    
            except Exception as e:
                logger.error(f"Error parsing item {idx}: {e}")
                continue
        
        return items
    
    async def _parse_html_response(self, html: str, url: str = None) -> List[Dict[str, Any]]:
        """解析HTML响应
        
        Args:
            html: HTML内容
            url: 请求URL
            
        Returns:
            解析后的数据列表
        """
        soup = BeautifulSoup(html, 'html.parser')
        items = []
        
        # 查找热榜列表
        hot_list = soup.select('.HotList-list .HotItem')
        
        for idx, item_elem in enumerate(hot_list, 1):
            try:
                # 提取标题
                title_elem = item_elem.select_one('.HotItem-content a')
                if not title_elem:
                    continue
                
                title = title_elem.get_text(strip=True)
                url = title_elem.get('href', '')
                
                # 提取热度
                metrics_elem = item_elem.select_one('.HotItem-metrics')
                hot_value = 0
                if metrics_elem:
                    hot_text = metrics_elem.get_text(strip=True)
                    # 提取数字
                    import re
                    numbers = re.findall(r'\d+', hot_text.replace(',', ''))
                    if numbers:
                        hot_value = int(numbers[0])
                
                # 提取摘要
                excerpt_elem = item_elem.select_one('.HotItem-excerpt')
                summary = excerpt_elem.get_text(strip=True) if excerpt_elem else ''
                
                hot_item = {
                    'rank': idx,
                    'title': title,
                    'url': self._normalize_url(url),
                    'hot_value': hot_value,
                    'summary': summary,
                    'category': '热榜'
                }
                
                items.append(hot_item)
                
            except Exception as e:
                logger.error(f"Error parsing HTML item {idx}: {e}")
                continue
        
        return items
    
    def _parse_question_item(self, rank: int, item: Dict, target: Dict) -> Optional[Dict[str, Any]]:
        """解析问题类型的热榜项
        
        Args:
            rank: 排名
            item: 原始数据
            target: 目标数据
            
        Returns:
            格式化的数据
        """
        try:
            question_id = target.get('id', '')
            title = target.get('title', '')
            
            # 提取答案数和关注数
            answer_count = target.get('answer_count', 0)
            follower_count = target.get('follower_count', 0)
            comment_count = target.get('comment_count', 0)
            
            # 提取摘要
            excerpt = target.get('excerpt', '')
            
            # 构建URL
            url = f"https://www.zhihu.com/question/{question_id}"
            
            # 计算热度值
            detail_text = item.get('detail_text', '')
            hot_value = self._extract_hot_value(detail_text)
            
            return {
                'rank': rank,
                'title': title,
                'url': url,
                'hot_value': hot_value,
                'summary': excerpt,
                'category': '问答',
                'view_count': follower_count,
                'comment_count': answer_count,
                'extra_data': {
                    'question_id': question_id,
                    'answer_count': answer_count,
                    'follower_count': follower_count
                }
            }
        except Exception as e:
            logger.error(f"Error parsing question item: {e}")
            return None
    
    def _parse_answer_item(self, rank: int, item: Dict, target: Dict) -> Optional[Dict[str, Any]]:
        """解析回答类型的热榜项
        
        Args:
            rank: 排名
            item: 原始数据
            target: 目标数据
            
        Returns:
            格式化的数据
        """
        try:
            answer_id = target.get('id', '')
            question = target.get('question', {})
            question_id = question.get('id', '')
            title = question.get('title', '')
            
            # 提取作者信息
            author = target.get('author', {})
            author_name = author.get('name', '')
            author_url = author.get('url', '')
            
            # 提取互动数据
            voteup_count = target.get('voteup_count', 0)
            comment_count = target.get('comment_count', 0)
            
            # 提取摘要
            excerpt = target.get('excerpt', '')
            
            # 构建URL
            url = f"https://www.zhihu.com/question/{question_id}/answer/{answer_id}"
            
            # 计算热度值
            detail_text = item.get('detail_text', '')
            hot_value = self._extract_hot_value(detail_text)
            
            return {
                'rank': rank,
                'title': title,
                'url': url,
                'hot_value': hot_value,
                'summary': excerpt,
                'category': '回答',
                'author': author_name,
                'author_url': self._normalize_url(author_url),
                'like_count': voteup_count,
                'comment_count': comment_count,
                'extra_data': {
                    'answer_id': answer_id,
                    'question_id': question_id
                }
            }
        except Exception as e:
            logger.error(f"Error parsing answer item: {e}")
            return None
    
    def _parse_article_item(self, rank: int, item: Dict, target: Dict) -> Optional[Dict[str, Any]]:
        """解析文章类型的热榜项
        
        Args:
            rank: 排名
            item: 原始数据
            target: 目标数据
            
        Returns:
            格式化的数据
        """
        try:
            article_id = target.get('id', '')
            title = target.get('title', '')
            
            # 提取作者信息
            author = target.get('author', {})
            author_name = author.get('name', '')
            author_url = author.get('url', '')
            
            # 提取互动数据
            voteup_count = target.get('voteup_count', 0)
            comment_count = target.get('comment_count', 0)
            
            # 提取摘要
            excerpt = target.get('excerpt', '')
            
            # 提取图片
            image_url = target.get('image_url', '')
            
            # 构建URL
            url = target.get('url', f"https://zhuanlan.zhihu.com/p/{article_id}")
            
            # 计算热度值
            detail_text = item.get('detail_text', '')
            hot_value = self._extract_hot_value(detail_text)
            
            return {
                'rank': rank,
                'title': title,
                'url': self._normalize_url(url),
                'hot_value': hot_value,
                'summary': excerpt,
                'category': '文章',
                'author': author_name,
                'author_url': self._normalize_url(author_url),
                'image_url': image_url,
                'like_count': voteup_count,
                'comment_count': comment_count,
                'extra_data': {
                    'article_id': article_id
                }
            }
        except Exception as e:
            logger.error(f"Error parsing article item: {e}")
            return None
    
    def _parse_generic_item(self, rank: int, item: Dict, target: Dict) -> Optional[Dict[str, Any]]:
        """解析通用类型的热榜项
        
        Args:
            rank: 排名
            item: 原始数据
            target: 目标数据
            
        Returns:
            格式化的数据
        """
        try:
            title = target.get('title', '') or target.get('topic', {}).get('name', '')
            url = target.get('url', '') or target.get('link', {}).get('url', '')
            
            # 提取摘要
            excerpt = target.get('excerpt', '') or target.get('description', '')
            
            # 计算热度值
            detail_text = item.get('detail_text', '')
            hot_value = self._extract_hot_value(detail_text)
            
            return {
                'rank': rank,
                'title': title,
                'url': self._normalize_url(url),
                'hot_value': hot_value,
                'summary': excerpt,
                'category': target.get('type', '其他')
            }
        except Exception as e:
            logger.error(f"Error parsing generic item: {e}")
            return None
    
    def _extract_hot_value(self, detail_text: str) -> int:
        """从详情文本中提取热度值
        
        Args:
            detail_text: 详情文本
            
        Returns:
            热度值
        """
        if not detail_text:
            return 0
        
        import re
        # 提取数字和单位
        pattern = r'([\d.]+)\s*(万|亿)?'
        match = re.search(pattern, detail_text)
        
        if match:
            number = float(match.group(1))
            unit = match.group(2)
            
            if unit == '万':
                return int(number * 10000)
            elif unit == '亿':
                return int(number * 100000000)
            else:
                return int(number)
        
        return 0
    
    def _normalize_url(self, url: str) -> str:
        """规范化URL
        
        Args:
            url: 原始URL
            
        Returns:
            规范化的URL
        """
        if not url:
            return ''
        
        if url.startswith('//'):
            return f"https:{url}"
        elif url.startswith('/'):
            return f"https://www.zhihu.com{url}"
        elif not url.startswith('http'):
            return f"https://www.zhihu.com/{url}"
        
        return url
    
    async def fetch_hot_list(self) -> HotItemBatch:
        """获取知乎热榜
        
        Returns:
            热榜批次数据
        """
        batch = HotItemBatch(platform=Platform.ZHIHU)
        
        # 使用Billboard页面（不需要认证）
        url = config_manager.get_api_url('zhihu', 'billboard')
        if not url:
            url = config_manager.get_api_url('zhihu', 'hot_list')
        
        # 爬取数据
        items = await self.crawl(url)
        
        # 转换为HotItem对象
        for item_data in items:
            try:
                # 生成数据ID
                item_data['data_id'] = self.data_normalizer.generate_id(item_data, 'zhihu')
                item_data['platform'] = Platform.ZHIHU
                
                # 判断热度状态
                if item_data.get('rank', 999) <= 3:
                    item_data['status'] = HotItemStatus.EXPLOSIVE
                elif item_data.get('rank', 999) <= 10:
                    item_data['status'] = HotItemStatus.HOT
                else:
                    item_data['status'] = HotItemStatus.RISING
                
                hot_item = HotItem(**item_data)
                
                if hot_item.is_valid():
                    batch.add_item(hot_item)
                    
            except Exception as e:
                logger.error(f"Error creating HotItem: {e}")
                continue
        
        logger.info(f"Fetched {batch.total_count} items from Zhihu")
        
        return batch
    
    async def fetch_with_details(self, limit: int = 10) -> HotItemBatch:
        """获取热榜并补充详情
        
        Args:
            limit: 获取数量限制
            
        Returns:
            包含详情的热榜数据
        """
        # 先获取基础热榜
        batch = await self.fetch_hot_list()
        
        # 获取前N个热点的详情
        top_items = batch.get_top_items(limit)
        
        for item in top_items:
            try:
                # 根据类型获取详情
                if 'question_id' in item.extra_data:
                    await self._fetch_question_details(item)
                elif 'answer_id' in item.extra_data:
                    await self._fetch_answer_details(item)
                elif 'article_id' in item.extra_data:
                    await self._fetch_article_details(item)
                    
            except Exception as e:
                logger.error(f"Error fetching details for {item.title}: {e}")
                continue
        
        return batch
    
    async def _fetch_question_details(self, item: HotItem):
        """获取问题详情
        
        Args:
            item: 热点项
        """
        question_id = item.extra_data.get('question_id')
        if not question_id:
            return
        
        url = config_manager.get_api_url('zhihu', 'question', question_id=question_id)
        if not url:
            return
        
        data = await self.fetch_json(url)
        if not data:
            return
        
        # 更新详情
        item.content = data.get('detail', '')
        item.view_count = data.get('view_count', item.view_count)
        item.comment_count = data.get('comment_count', item.comment_count)
        
        # 提取标签
        topics = data.get('topics', [])
        item.tags = [topic.get('name', '') for topic in topics if topic.get('name')]
    
    async def _fetch_answer_details(self, item: HotItem):
        """获取回答详情
        
        Args:
            item: 热点项
        """
        # 实现回答详情获取逻辑
        pass
    
    async def _fetch_article_details(self, item: HotItem):
        """获取文章详情
        
        Args:
            item: 热点项
        """
        # 实现文章详情获取逻辑
        pass